# Replication files for:
# "Generalizability of Heterogeneous Treatment Effect Estimates Across Samples"
# Alexander Coppock, Thomas J. Leeper, and Kevin J. Mullinix
# Proceedings of the National Academy of Sciences, Forthcoming

rm(list = ls())

library(dplyr)
library(readr)

# load data ---------------------------------------------------------------

scatter_df <- read_rds("CLM_scatter_df.rds")

# In text numbers ---------------------------------------------------------

# Out of 393 opportunities, the difference-in-CATEs is significant 59 times, or 15\% of the time.
dic_sig_table <- table(scatter_df$`Difference in CATES`)
dic_sig_table
dic_sig_table[1] / sum(dic_sig_table)

# In zero of 393 opportunities do the CATEs have different signs while both being statistically significant.

scatter_df <-
  scatter_df %>%
  mutate(disagree =
           (sign(estimate_mt) != sign(estimate_original)) &
           (sig_mt == "Significant") &
           (sig_original == "Significant"))

table(scatter_df$disagree)

# Of the 156 CATEs that were significantly different from no effect in the original, 118 are significantly different from no effect in the MTurk replication.

# Of the 237 CATEs that were statistically indistinguishable from no effect in the original, 158 were statistically indistinguishable from zero in the MTurk version.

sig_table <- with(scatter_df, table(sig_mt, sig_original))
colSums(sig_table)
sig_table

sum(diag(sig_table))/sum(sig_table)

#
scatter_df %>% summarise(mt_original = cor(estimate_mt, estimate_original, use = "complete.obs"))
